"""
This is the X-Spider base class
"""
import importlib
import re
from celery import Task
from rlibs.base import XSession
from w3lib.url import canonicalize_url
from requests_futures.sessions import FuturesSession
from rlibs.file_saver import File
from urllib.parse import urlparse
from rlibs.filter import UrlDuplicate
from hashlib import sha1


class BaseSpider(Task):
    """
    base spider class for all the spiders
    """
    def __init__(self, **kwargs):
        self.session = XSession()
        file_path = self.app.conf.get('FILE_PATH')
        if urlparse(file_path).scheme == 'local':
            self.file_saver = File(base_path=urlparse(file_path).path)
        print(self.app.main)
        uri = self.app.conf.get('REDIS_URI')
        print('initiate redis uri', uri)
        self.url_filter = UrlDuplicate(uri=uri)
        print('finished redis uri', uri)

    @property
    def reset_session(self):
        self.session = XSession()

    def x_middle_ware(self, x_request_result=None, middle_ware=None):
        if middle_ware and isinstance(middle_ware, list):
            middle_class = []
            for i in middle_ware:
                module_name, class_name = i.rsplit(".", 1)
                module = importlib.import_module(module_name)
                if hasattr(module, class_name):
                    middle_class.append(
                        getattr(module, class_name)
                    )
                else:
                    print(module_name, 'dont have', class_name)
            process_lists = [getattr(i, 'process_request') for i in
                             middle_class if hasattr(i, 'process_request')]
 
            for i in process_lists:
                x_request_result = i(x_request_result)
            return x_request_result

    def x_request(self, url, method='get', **kwargs):
        """
        custom request for spiders
        """
        if hasattr(self.session, method):
            req = getattr(self.session, method)
            try:
                result = req(url, **kwargs)
            except Exception as tmp:
                result = tmp
            return result
        else:
            raise Exception('request doesnt have the {0} method'.format(method))

    def handle_url(self, url):
        """
        handle url to the same order.
        """
        return canonicalize_url(url)

    def check_url(self, key, url):
        url = self.handle_url(url)
        return self.url_filter.check(key, url)

    def limit_url(self, url, limits=None):
        if not limits:
            limits = []
        if urlparse(url).netloc in limits:
            return url
        else:
            return None
